MFC r298294

1. modify fwdump (a.k.a grcdump) so that grcdump memory is allocated
     and freed on as needed basis.
2. grcdump can be taken at failure points by invoking bxe_grc_dump()
     when trigger_grcdump sysctl flag is set. When grcdump is taken
     grcdump_done sysctl flag is set.
3. grcdump_done can be monitored by the user to retrieve the grcdump

3 files changed, 169 insertions, 67 deletions

diff --git a/sys/dev/bxe/bxe.c b/sys/dev/bxe/bxe.c
index 2e0e3e936ecd..50f4c421102c 100644
--- a/sys/dev/bxe/bxe.c
+++ b/sys/dev/bxe/bxe.c
@@ -683,7 +683,6 @@ static void bxe_handle_fp_tq(void *context, int pending);
 
 static int bxe_add_cdev(struct bxe_softc *sc);
 static void bxe_del_cdev(struct bxe_softc *sc);
-static int bxe_grc_dump(struct bxe_softc *sc);
 static int bxe_alloc_buf_rings(struct bxe_softc *sc);
 static void bxe_free_buf_rings(struct bxe_softc *sc);
 
@@ -3460,6 +3459,10 @@ bxe_watchdog(struct bxe_softc    *sc,
     }
 
     BLOGE(sc, "TX watchdog timeout on fp[%02d], resetting!\n", fp->index);
+    if(sc->trigger_grcdump) {
+         /* taking grcdump */
+         bxe_grc_dump(sc);
+    }
 
     BXE_FP_TX_UNLOCK(fp);
 
@@ -15669,30 +15672,6 @@ bxe_sysctl_state(SYSCTL_HANDLER_ARGS)
 }
 
 static int
-bxe_sysctl_trigger_grcdump(SYSCTL_HANDLER_ARGS)
-{
-    struct bxe_softc *sc;
-    int error, result;
-
-    result = 0;
-    error = sysctl_handle_int(oidp, &result, 0, req);
-
-    if (error || !req->newptr) {
-        return (error);
-    }
-
-    if (result == 1) {
-        sc = (struct bxe_softc *)arg1;
-
-        BLOGI(sc, "... grcdump start ...\n");
-        bxe_grc_dump(sc);
-        BLOGI(sc, "... grcdump done ...\n");
-    }
-
-    return (error);
-}
-
-static int
 bxe_sysctl_eth_stat(SYSCTL_HANDLER_ARGS)
 {
     struct bxe_softc *sc = (struct bxe_softc *)arg1;
@@ -15843,14 +15822,16 @@ bxe_add_sysctls(struct bxe_softc *sc)
                     "debug logging mode");
 #endif /* #if __FreeBSD_version >= 900000 */
 
-    SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "trigger_grcdump",
-                    CTLTYPE_UINT | CTLFLAG_RW, sc, 0,
-                    bxe_sysctl_trigger_grcdump, "IU",
-                    "set by driver when a grcdump is needed");
+    sc->trigger_grcdump = 0;
+    SYSCTL_ADD_UINT(ctx, children, OID_AUTO, "trigger_grcdump",
+                   CTLFLAG_RW, &sc->trigger_grcdump, 0,
+                   "trigger grcdump should be invoked"
+                   "  before collecting grcdump");
 
+    sc->grcdump_started = 0;
     sc->grcdump_done = 0;
     SYSCTL_ADD_UINT(ctx, children, OID_AUTO, "grcdump_done",
-                   CTLFLAG_RW, &sc->grcdump_done, 0,
+                   CTLFLAG_RD, &sc->grcdump_done, 0,
                    "set by driver when grcdump is done");
 
     sc->rx_budget = bxe_rx_budget;
@@ -18682,7 +18663,7 @@ bxe_get_preset_regs(struct bxe_softc *sc, uint32_t *p, uint32_t preset)
     return 0;
 }
 
-static int
+int
 bxe_grc_dump(struct bxe_softc *sc)
 {
     int rval = 0;
@@ -18690,12 +18671,53 @@ bxe_grc_dump(struct bxe_softc *sc)
     uint8_t *buf;
     uint32_t size;
     struct  dump_header *d_hdr;
+    uint32_t i;
+    uint32_t reg_val;
+    uint32_t reg_addr;
+    uint32_t cmd_offset;
+    int context_size;
+    int allocated;
+    struct ecore_ilt *ilt = SC_ILT(sc);
+    struct bxe_fastpath *fp;
+    struct ilt_client_info *ilt_cli;
+    int grc_dump_size;
+
 
-    if (sc->grcdump_done)
+    if (sc->grcdump_done || sc->grcdump_started)
 	return (rval);
     
+    sc->grcdump_started = 1;
+    BLOGI(sc, "Started collecting grcdump\n");
+
+    grc_dump_size = (bxe_get_total_regs_len32(sc) * sizeof(uint32_t)) +
+                sizeof(struct  dump_header);
+
+    sc->grc_dump = malloc(grc_dump_size, M_DEVBUF, M_NOWAIT);
+
+    if (sc->grc_dump == NULL) {
+        BLOGW(sc, "Unable to allocate memory for grcdump collection\n");
+        return(ENOMEM);
+    }
+
+
+
+    /* Disable parity attentions as long as following dump may
+     * cause false alarms by reading never written registers. We
+     * will re-enable parity attentions right after the dump.
+     */
+
+    /* Disable parity on path 0 */
+    bxe_pretend_func(sc, 0);
+
+    ecore_disable_blocks_parity(sc);
+
+    /* Disable parity on path 1 */
+    bxe_pretend_func(sc, 1);
     ecore_disable_blocks_parity(sc);
 
+    /* Return to current function */
+    bxe_pretend_func(sc, SC_ABS_FUNC(sc));
+
     buf = sc->grc_dump;
     d_hdr = sc->grc_dump;
 
@@ -18727,7 +18749,7 @@ bxe_grc_dump(struct bxe_softc *sc)
             (preset_idx == 11))
             continue;
 
-        rval = bxe_get_preset_regs(sc, sc->grc_dump, preset_idx);
+        rval = bxe_get_preset_regs(sc, (uint32_t *)buf, preset_idx);
 
 	if (rval)
             break;
@@ -18737,9 +18759,81 @@ bxe_grc_dump(struct bxe_softc *sc)
         buf += size;
     }
 
+    bxe_pretend_func(sc, 0);
+    ecore_clear_blocks_parity(sc);
+    ecore_enable_blocks_parity(sc);
+
+    bxe_pretend_func(sc, 1);
     ecore_clear_blocks_parity(sc);
     ecore_enable_blocks_parity(sc);
 
+    /* Return to current function */
+    bxe_pretend_func(sc, SC_ABS_FUNC(sc));
+
+
+    context_size = (sizeof(union cdu_context) * BXE_L2_CID_COUNT(sc));
+    for (i = 0, allocated = 0; allocated < context_size; i++) {
+
+        BLOGI(sc, "cdu_context i %d paddr %#jx vaddr %p size 0x%zx\n", i,
+            (uintmax_t)sc->context[i].vcxt_dma.paddr,
+            sc->context[i].vcxt_dma.vaddr,
+            sc->context[i].size);
+        allocated += sc->context[i].size;
+    }
+    BLOGI(sc, "fw stats start_paddr %#jx end_paddr %#jx vaddr %p size 0x%x\n",
+        (uintmax_t)sc->fw_stats_req_mapping,
+        (uintmax_t)sc->fw_stats_data_mapping,
+        sc->fw_stats_req, (sc->fw_stats_req_size + sc->fw_stats_data_size));
+    BLOGI(sc, "def_status_block paddr %p vaddr %p size 0x%zx\n",
+        (void *)sc->def_sb_dma.paddr, sc->def_sb,
+        sizeof(struct host_sp_status_block));
+    BLOGI(sc, "event_queue paddr %#jx vaddr %p size 0x%x\n",
+        (uintmax_t)sc->eq_dma.paddr, sc->eq_dma.vaddr, BCM_PAGE_SIZE);
+    BLOGI(sc, "slow path paddr %#jx vaddr %p size 0x%zx\n",
+        (uintmax_t)sc->sp_dma.paddr, sc->sp_dma.vaddr,
+        sizeof(struct bxe_slowpath));
+    BLOGI(sc, "slow path queue paddr %#jx vaddr %p size 0x%x\n",
+        (uintmax_t)sc->spq_dma.paddr, sc->spq_dma.vaddr, BCM_PAGE_SIZE);
+    BLOGI(sc, "fw_buf paddr %#jx vaddr %p size 0x%x\n",
+        (uintmax_t)sc->gz_buf_dma.paddr, sc->gz_buf_dma.vaddr,
+        FW_BUF_SIZE);
+    for (i = 0; i < sc->num_queues; i++) {
+        fp = &sc->fp[i];
+        BLOGI(sc, "FP status block fp %d paddr %#jx vaddr %p size 0x%zx\n", i,
+            (uintmax_t)fp->sb_dma.paddr, fp->sb_dma.vaddr,
+            sizeof(union bxe_host_hc_status_block));
+        BLOGI(sc, "TX BD CHAIN fp %d paddr %#jx vaddr %p size 0x%x\n", i,
+            (uintmax_t)fp->tx_dma.paddr, fp->tx_dma.vaddr,
+            (BCM_PAGE_SIZE * TX_BD_NUM_PAGES));
+        BLOGI(sc, "RX BD CHAIN fp %d paddr %#jx vaddr %p size 0x%x\n", i,
+            (uintmax_t)fp->rx_dma.paddr, fp->rx_dma.vaddr,
+            (BCM_PAGE_SIZE * RX_BD_NUM_PAGES));
+        BLOGI(sc, "RX RCQ CHAIN fp %d paddr %#jx vaddr %p size 0x%zx\n", i,
+            (uintmax_t)fp->rcq_dma.paddr, fp->rcq_dma.vaddr,
+            (BCM_PAGE_SIZE * RCQ_NUM_PAGES));
+        BLOGI(sc, "RX SGE CHAIN fp %d paddr %#jx vaddr %p size 0x%x\n", i,
+            (uintmax_t)fp->rx_sge_dma.paddr, fp->rx_sge_dma.vaddr,
+            (BCM_PAGE_SIZE * RX_SGE_NUM_PAGES));
+    }
+
+    ilt_cli = &ilt->clients[1];
+    for (i = ilt_cli->start; i <= ilt_cli->end; i++) {
+        BLOGI(sc, "ECORE_ILT paddr %#jx vaddr %p size 0x%x\n",
+            (uintmax_t)(((struct bxe_dma *)((&ilt->lines[i])->page))->paddr),
+            ((struct bxe_dma *)((&ilt->lines[i])->page))->vaddr, BCM_PAGE_SIZE);
+    }
+
+
+    cmd_offset = DMAE_REG_CMD_MEM;
+    for (i = 0; i < 224; i++) {
+        reg_addr = (cmd_offset +(i * 4));
+        reg_val = REG_RD(sc, reg_addr);
+        BLOGI(sc, "DMAE_REG_CMD_MEM i=%d reg_addr 0x%x reg_val 0x%08x\n",i,
+            reg_addr, reg_val);
+    }
+
+
+    BLOGI(sc, "Collection of grcdump done\n");
     sc->grcdump_done = 1;
     return(rval);
 }
@@ -18747,21 +18841,10 @@ bxe_grc_dump(struct bxe_softc *sc)
 static int
 bxe_add_cdev(struct bxe_softc *sc)
 {
-    int grc_dump_size;
-
-    grc_dump_size = (bxe_get_total_regs_len32(sc) * sizeof(uint32_t)) +
-				sizeof(struct  dump_header);
-
-    sc->grc_dump = malloc(grc_dump_size, M_DEVBUF, M_NOWAIT);
-
-    if (sc->grc_dump == NULL)
-        return (-1);
-
     sc->eeprom = malloc(BXE_EEPROM_MAX_DATA_LEN, M_DEVBUF, M_NOWAIT);
 
     if (sc->eeprom == NULL) {
         BLOGW(sc, "Unable to alloc for eeprom size buffer\n");
-        free(sc->grc_dump, M_DEVBUF); sc->grc_dump = NULL;
         return (-1);
     }
 
@@ -18774,11 +18857,8 @@ bxe_add_cdev(struct bxe_softc *sc)
                             if_name(sc->ifnet));
 
     if (sc->ioctl_dev == NULL) {
-
-        free(sc->grc_dump, M_DEVBUF);
         free(sc->eeprom, M_DEVBUF);
         sc->eeprom = NULL;
-
         return (-1);
     }
 
@@ -18793,13 +18873,11 @@ bxe_del_cdev(struct bxe_softc *sc)
     if (sc->ioctl_dev != NULL)
         destroy_dev(sc->ioctl_dev);
 
-    if (sc->grc_dump != NULL)
-        free(sc->grc_dump, M_DEVBUF);
-
     if (sc->eeprom != NULL) {
         free(sc->eeprom, M_DEVBUF);
         sc->eeprom = NULL;
     }
+    sc->ioctl_dev = NULL;
 
     return;
 }
@@ -18977,15 +19055,26 @@ bxe_eioctl(struct cdev *dev, u_long cmd, caddr_t data, int fflag,
             
             grc_dump_size = (bxe_get_total_regs_len32(sc) * sizeof(uint32_t)) +
                                 sizeof(struct  dump_header);
-
-            if ((sc->grc_dump == NULL) || (dump->grcdump == NULL) ||
-                (dump->grcdump_size < grc_dump_size) || (!sc->grcdump_done)) {
+            if ((!sc->trigger_grcdump) || (dump->grcdump == NULL) ||
+                (dump->grcdump_size < grc_dump_size)) {
                 rval = EINVAL;
                 break;
             }
-	    dump->grcdump_dwords = grc_dump_size >> 2;
-            rval = copyout(sc->grc_dump, dump->grcdump, grc_dump_size);
-            sc->grcdump_done = 0;
+
+            if((sc->trigger_grcdump) && (!sc->grcdump_done) &&
+                (!sc->grcdump_started)) {
+                rval =  bxe_grc_dump(sc);
+            }
+
+            if((!rval) && (sc->grcdump_done) && (sc->grcdump_started) &&
+                (sc->grc_dump != NULL))  {
+                dump->grcdump_dwords = grc_dump_size >> 2;
+                rval = copyout(sc->grc_dump, dump->grcdump, grc_dump_size);
+                free(sc->grc_dump, M_DEVBUF);
+                sc->grc_dump = NULL;
+                sc->grcdump_started = 0;
+                sc->grcdump_done = 0;
+            }
 
             break;
 
@@ -19005,6 +19094,7 @@ bxe_eioctl(struct cdev *dev, u_long cmd, caddr_t data, int fflag,
             snprintf(drv_infop->bus_info, BXE_BUS_INFO_LENGTH, "%d:%d:%d",
                 sc->pcie_bus, sc->pcie_device, sc->pcie_func);
             break;
+
         case BXE_DEV_SETTING:
             dev_p = (bxe_dev_setting_t *)data;
             bxe_get_settings(sc, &dev_set);
@@ -19023,20 +19113,20 @@ bxe_eioctl(struct cdev *dev, u_long cmd, caddr_t data, int fflag,
             reg_p = (bxe_get_regs_t *)data;
             grc_dump_size = reg_p->reg_buf_len;
 
-            if (sc->grc_dump == NULL) { 
-                rval = EINVAL;
-                break;
-            }
-
-            if(!sc->grcdump_done) {
+            if((!sc->grcdump_done) && (!sc->grcdump_started)) {
                 bxe_grc_dump(sc);
             }
-            if(sc->grcdump_done) {
+            if((sc->grcdump_done) && (sc->grcdump_started) &&
+                (sc->grc_dump != NULL))  {
                 rval = copyout(sc->grc_dump, reg_p->reg_buf, grc_dump_size);
+                free(sc->grc_dump, M_DEVBUF);
+                sc->grc_dump = NULL;
+                sc->grcdump_started = 0;
                 sc->grcdump_done = 0;
             }
 
             break;
+
         case BXE_RDW_REG:
             reg_rdw_p = (bxe_reg_rdw_t *)data;
             if((reg_rdw_p->reg_cmd == BXE_READ_REG_CMD) &&
diff --git a/sys/dev/bxe/bxe.h b/sys/dev/bxe/bxe.h
index 056a89211c34..ae98e9c55867 100644
--- a/sys/dev/bxe/bxe.h
+++ b/sys/dev/bxe/bxe.h
@@ -1786,8 +1786,12 @@ struct bxe_softc {
     int panic;
 
     struct cdev *ioctl_dev;
+
     void *grc_dump;
-    int grcdump_done;
+    unsigned int trigger_grcdump;
+    unsigned int  grcdump_done;
+    unsigned int grcdump_started;
+
     void *eeprom;
 }; /* struct bxe_softc */
 
@@ -2293,7 +2297,7 @@ void bxe_dump_mem(struct bxe_softc *sc, char *tag,
                   uint8_t *mem, uint32_t len);
 void bxe_dump_mbuf_data(struct bxe_softc *sc, char *pTag,
                         struct mbuf *m, uint8_t contents);
-
+extern int bxe_grc_dump(struct bxe_softc *sc);
 
 #if __FreeBSD_version >= 800000
 #if __FreeBSD_version >= 1000000
diff --git a/sys/dev/bxe/bxe_stats.c b/sys/dev/bxe/bxe_stats.c
index 4a0ad192922d..f75ea8251fd5 100644
--- a/sys/dev/bxe/bxe_stats.c
+++ b/sys/dev/bxe/bxe_stats.c
@@ -234,6 +234,10 @@ bxe_stats_comp(struct bxe_softc *sc)
     while (*stats_comp != DMAE_COMP_VAL) {
         if (!cnt) {
             BLOGE(sc, "Timeout waiting for stats finished\n");
+            if(sc->trigger_grcdump) {
+                /* taking grcdump */
+                bxe_grc_dump(sc);
+            }
             break;
         }
 
@@ -1310,8 +1314,12 @@ bxe_stats_update(struct bxe_softc *sc)
         if (bxe_storm_stats_update(sc)) {
             if (sc->stats_pending++ == 3) {
 		if (sc->ifnet->if_drv_flags & IFF_DRV_RUNNING) {
-			atomic_store_rel_long(&sc->chip_tq_flags, CHIP_TQ_REINIT);
-			taskqueue_enqueue(sc->chip_tq, &sc->chip_tq_task);
+                    if(sc->trigger_grcdump) {
+                        /* taking grcdump */
+                        bxe_grc_dump(sc);
+                    }
+                    atomic_store_rel_long(&sc->chip_tq_flags, CHIP_TQ_REINIT);
+                    taskqueue_enqueue(sc->chip_tq, &sc->chip_tq_task);
 		}
             }
             return;